# -*- coding: UTF-8 -*-
# Spider for 'etsy.com'
#Author: Denis <sinedone@gmail.com>,

from urlparse import urljoin

from scrapy.spider import BaseSpider
from scrapy.http import  Request
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.loader import XPathItemLoader
from scrapy.contrib.loader.processor import TakeFirst, Join
from scrapy.shell import inspect_response
from asos.items import ClothItem

class Etsy(BaseSpider):

    name = 'etsy'
    start_urls = ['http://www.etsy.com/categories']
    
    URL = 'http://www.etsy.com'
    
    def parse(self, response):
#        inspect_response(response)
        hxs = HtmlXPathSelector(response)
        for cat in hxs.select("//div[@class='category']"):
            category = cat.select(".//h2/a/text()").extract()[0]
            for subcat in cat.select("div//a"):
                subcategory = subcat.select("text()").extract()[0]
                url = self.URL + subcat.select("@href").extract()[0] 
                yield Request(url, meta={'category':u'%s - %s' %(category,subcategory)},
                              callback=self.list_subcats)
    
    def list_subcats(self, response):
        hxs = HtmlXPathSelector(response)
        category = response.request.meta['category']
        for cat in hxs.select("//ul[@id='category-nav']/li[@class='active']//li/a"):
            subcategory = cat.select("text()").extract()[0]
            url = self.URL + cat.select("@href").extract()[0]
            yield Request(url, meta={'category':u"%s - %s" %(category, subcategory)},
                          callback=self.list_items)
        else:
            for i in self.list_items(response):
                yield i
    
    def list_items(self, response):
        hxs = HtmlXPathSelector(response)
        category = response.request.meta['category']
#        sex = response.request.meta['sex']
        for item in hxs.select("//li[@class='listing-card']"):
            url = item.select("a/@href").extract()[0]
            price = item.select(".//p[@class='listing-price']/text()").extract()[0]
            yield Request(url, meta={'category':category, 'price':price},
                          dont_filter=True, callback=self.get_item)
        next_page = hxs.select("//div[class='pager']//a[@class='next']/@href").extract()
        if next_page:
            yield Request(next_page, meta={'category':category},
                          callback=self.list_items)
            
    def get_item(self, response):
        hxs = HtmlXPathSelector(response)
        l = XPathItemLoader(ClothItem(), hxs)
        l.add_value('url', [unicode(response.url)])
        l.add_value('category', [response.request.meta['category']])
#        l.add_value('sex', [response.request.meta['sex']])
        l.add_value('price', [response.request.meta['price']])
        l.add_xpath('title', "//h1/text()", TakeFirst())
        l.add_xpath('designer', "//li[@class='member']/a/text()", TakeFirst())
        l.add_xpath('description', "//div[@class='section-content']/text()", Join(u'\r\n'))
        l.add_xpath('facebookURL', "//like/@href", TakeFirst())
        for i, image in enumerate(hxs.select("//div[@id='item-thumbs']/a/@href").re(
                         r"image_id=(\d+)$")[:4]):
            l.add_value('image%s' %str(i+1),
                        ['http://ny-image0.etsy.com/il_fullxfull.%s.jpg' %image])
        return l.load_item()
        
        
        
        
        
        